//
package main

import (
	"gitee.com/congqian/dense-spider/core/common"

	"gitee.com/congqian/dense-spider/core/common/extract"
	"gitee.com/congqian/dense-spider/core/common/filter"
	"gitee.com/congqian/dense-spider/core/history"
	"gitee.com/congqian/dense-spider/core/page_processer"
	"gitee.com/congqian/dense-spider/core/spider"
	"gitee.com/congqian/dense-spider/extension/pipeline"
)

func main() {
	// spider input:
	//  PageProcesser ;
	//  task name used in Pipeline for record;
	ex := make([]extract.ExtractRule, 0)
	ex = append(ex, extract.NewExtractRule("Title", "//*[@id=\"cb_post_title_url\"]", "text"))
	ex = append(ex, extract.NewExtractRule("Content", "//*[@id=\"cnblogs_post_body\"]", "html"))
	sp := spider.NewSpider(page_processer.NewHtmlPageProcesser(), "cnblogs", "https://www.cnblogs.com/jackyfei/", ex, common.UseAgents, "./logs/", nil)
	sp.SetMaxCrawlDepth(5)
	sp.SetThreadnum(4)

	sp.AddExcludeFilter(filter.NewRegexFilter("(\\.jpg|\\.css|\\.js|\\.gif|\\.jpeg|\\.png|\\.ico)"))

	sp.AddIncludeFilter(filter.NewRegexFilter("www.cnblogs.com/jackyfei/"))
	//sp.SetDownloader(downloader.NewChromeDownloader(nil))
	sp.SetHistory(history.NewBloomHistory("./data/history.bf"))
	// GetWithParams Params:
	//  1. Url.
	//  2. Responce type is "html" or "json" or "jsonp" or "text".
	//  3. The urltag is name for marking url and distinguish different urls in PageProcesser and Pipeline.
	//  4. The method is POST or GET.
	//  5. The postdata is body string sent to sever.
	//  6. The header is header for http request.
	//  7. Cookies

	sp.AddPipeline(pipeline.NewPipelineDb())
	sp.Run()
	println("-----------------------------------spider.Get---------------------------------")

}
